package lion.spider;

import java.util.ArrayList;
import java.util.List;

import lion.crawl.Traveler;
import lion.crawl.parse.URLRule;
import lion.crawl.rule.JsoupRule;
import lion.crawl.rule.ParseRule;
import lion.crawl.rule.RegexFieldRule;
import lion.dev.lang.MapJ;
import lion.framework.core.db.DB;
import lion.spider.anno.ParseHandlerManager;
import lion.spider.model.CrawlerConfig;
import lion.spider.model.ExtractRule;
import lion.spider.model.PageRule;

/**
 * 
 * 爬虫控制类
 * 
 * @author lion
 * @email hl_0074@sina.com
 * @date 2016年5月6日
 */
public class SpiderTravler {

	/**
	 * 对爬虫进行初始化设置后，启动爬虫
	 */
	public void start(CrawlerConfig config) {

		Traveler traveler = new Traveler();

		if (config.isCrawled()) {
			traveler.setLevel(config.getLevel());
		} else {
			traveler.setLevel(config.getFirstlevel());
		}

		traveler.setName(config.getName());
		traveler.setThreadNum(2);
		traveler.setStartPage(config.getStartPage());

		traveler.setUrlBase(new SpiderURLBase(config));

		List<URLRule> pageUrles = new ArrayList<URLRule>();
		List<ParseRule> rules = new ArrayList<ParseRule>();
		try (DB db = new DB()) {
			List<PageRule> prules = PageRule.list(-1, db);
			List<ExtractRule> extractRulesRules = ExtractRule.list(-1, db);

			for (PageRule pr : prules) {
				pageUrles.add(new URLRule(pr.getPattern(), pr.isParse()));
			}
			MapJ jsoupMap = new MapJ();
			for (ExtractRule er : extractRulesRules) {
				if (er.getType() == 1) {
					rules.add(new RegexFieldRule(er.getName(), er.getRule()));
				} else if (er.getType() == 2) {
					jsoupMap.put(er.getName(), er.getRule());
				}
			}
			if (!jsoupMap.isEmpty()) {
				rules.add(new JsoupRule(jsoupMap));
			}
		}

		// no filed to extract
		if (rules.isEmpty()) { throw new RuntimeException("parse rule is empty"); }
		traveler.setPageRules(pageUrles);
		traveler.setParseRules(rules);

		// 抓取结果handler
		IParseHandler handler = ParseHandlerManager.getHandler(config.getHandler());
		if (handler == null) { throw new RuntimeException("handler not found for name:" + config.getHandler()); }
		traveler.setParseListener(new SpiderListener(handler));

		traveler.start();
	}
}
